﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Text.RegularExpressions;
using System.Net;
using System.Collections;
using System.Threading;
using System.IO;

namespace 爬虫
{
    internal class Crawler
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        //static void Main(string[] args)
        //{
        //    Crawler crawler = new Crawler();
        //    string startUrl = "https://www.cnblogs.com/dstang2000/";
        //    if (args.Length >= 1) startUrl = args[0];
        //    crawler.urls.Add(startUrl, false);
        //    new Thread(crawler.Crawl).Start();
        //    Console.Read();
        //}
        public string Crawl(string start)
        {
            urls.Clear();
            count = 0;
            urls[start] = false;
            string ret = "";
            Console.WriteLine("开始爬行了....");
            while (true)
            {
                
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url]) continue;
                    current = url;
                }
                if (current == null || count > 10) break;
                ret = ret + current + "\n";
                Console.WriteLine("爬行" + current + "页面！");
                string html = DownLoad(current);
                urls[current] = true;
                count++;
                Parse(html, current);
            }
            Console.WriteLine("爬行结束");
            return ret;
        }
        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                string filename = count.ToString();
                File.WriteAllText(filename, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }
        public void Parse(string html, string url)
        {
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            string check = @"[\w]*\.html";
            Regex regex = new Regex(check);
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                //strRef = @"(href | HREF) []* = []* [""'][^""'#>]+ [""']";
                strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ' ', '>');
                if (strRef.Length == 0) continue;
                if (regex.IsMatch(strRef))
                {
                    try
                    {
                        string a = reset(strRef, url);
                        int b = url.IndexOf(".com");
                        int c = a.IndexOf(".com");
                        int d = url.IndexOf(":");
                        int e = a.IndexOf(":");
                        if (url.Substring(d, b - d) != a.Substring(e, c - e)) continue;
                        if(urls[a]==null) urls[a] = false;

                    }
                    catch
                    {

                    }
                }   
                
            }
        }
        public string reset(string strRef, string url)
        {
            if (strRef.StartsWith("http")) return strRef;

            else if (strRef.StartsWith("//"))
                return "https:" + strRef;
            else if (strRef.StartsWith("/"))
            {
                int t = url.IndexOf(".com");
                string a = url.Substring(0, t + 4);
                return a + strRef;
            }
            else if (strRef.StartsWith("./"))
            {
                if (url.EndsWith("/"))
                {
                    return url + strRef.Substring(2);
                }
                else return url + strRef.Substring(1);
            }
            else if (strRef.StartsWith("../"))
            {
                int n = url.Length;
                string s = url;
                if (s.EndsWith("/"))
                {
                    s = url.Substring(0, n - 1);
                }
                int t = s.LastIndexOf("/");
                s = s.Substring(0, t);
                return url + strRef.Substring(2);
            }
            return url;
        }

    }
}
